package engine;

import java.util.StringTokenizer;

/**
 * Tokenizer
 * 
 * @author ak2765
 * @author Vika
 *
 */
public final class WordTokenizer {

	private static String WORD_DELIMITERS = " \n\t.,:'\"()?!";
	
	/**
	 * @param sentence
	 * @return
	 */
	public static StringTokenizer tokenize(String sentence) {
		assert sentence != null : "sentence is null";
		
		return new StringTokenizer(sentence.toLowerCase(), WORD_DELIMITERS);
	}

}